#ifndef HSE_TOKENIZER_H
#define HSE_TOKENIZER_H
#include "hse_common.h"
#include <stdio.h>
#include <string>

namespace hse {

using std::vector;
using std::string;

class Tokenizer {
public:
	inline static bool IsSpliter(unsigned char c) {return is_spilter_[c];}
	static void Tokenize(string& query, vector<string>& tokens, int max_token = MAX_TOKEN_LEN);
	static void Init();
private:
	static bool is_spilter_[256];

	DISALLOW_COPY_AND_ASSIGN(Tokenizer);
};

}

#endif